# 导入相关库
from sklearn.model_selection import cross_val_score
from lightgbm.sklearn import LGBMRegressor
from sklearn.metrics import mean_absolute_error,  make_scorer
import catboost as cb
from nni.algorithms.feature_engineering.gbdt_selector import GBDTSelector
import lightgbm as lgb
import pandas as pd
import numpy as np
cat = {
    'model': 'category',
    'brand': 'category',
    'bodyType': 'category',
    'fuelType': 'category',
    'gearbox': 'category',
    'notRepairedDamage': 'category',
    'regionCode': 'category',
    'seller': 'category',
    'offerType': 'category',
}

#----------导入数据----------
test = pd.read_csv('./data/used_car_testB_20200421.csv', sep=' ', dtype=cat)
train = pd.read_csv('./data/used_car_train_20200313.csv', sep=' ', dtype=cat)
train['train'] = 1
test['train'] = 0
df = pd.concat([train, test], ignore_index=True)
#----------格式转换----------
df['notRepairedDamage'].replace('-', np.nan, inplace=True)


def to_dt(x):
    m = int(x[4:6])
    if m == 0:
        m = 1
    return x[:4] + '-' + str(m) + '-' + x[6:]


df['regDate'] = pd.to_datetime(df['regDate'].astype('str').apply(to_dt))
df['creatDate'] = pd.to_datetime(df['creatDate'].astype('str').apply(to_dt))
df.regDate = df.regDate.astype(int)
df.creatDate = df.creatDate.astype(int)
df['age'] = df.regDate-df.creatDate
'''
for i in ['model', 'bodyType', 'fuelType', 'gearbox', 'notRepairedDamage', 'price']:
    for j in df[df[i].isna()].index:
        try:
            df.loc[j, i] = df[df['name'] == df.loc[j, 'name']][i].mode()[0]
        except:
            pass
'''
#----------删除异常----------
del df['name']
del df['offerType']
del df['seller']
df['power'][df['power'] > 600] = 600
df['power'][df['power'] < 1] = 1
df['v_13'][df['v_13'] > 6] = 6
df['v_14'][df['v_14'] > 4] = 4


#----------填充缺失----------
def predict_na(data, cname):
    train = data.dropna()
    test = data[data[cname].isna()].agg(
        lambda x: x.fillna(x.mode())
    )
    test.drop(cname, axis=1, inplace=True)
    model = lgb.LGBMClassifier(
        boosting_type='gbdt'
    )
    #训练模型
    model.fit(
        train.drop(cname, axis=1),
        train[cname]
    )
    #对测试集进行预测
    p = model.predict(test)
    for i, j in enumerate(test.index):
        data.loc[j, cname] = p[i]
    return data[cname]


data0 = df.drop(['SaleID', 'price', 'train', 'regionCode'],
                axis=1)  # 剔除功能性特征与label值
lna = data0.isna().sum()[data0.isna().sum() != 0].index
for i in lna:
    p = pd.Categorical(predict_na(data0, i))
    data0[i] = p
    df[i] = p
#----------存储结果----------
df.to_csv('./user_data/df.csv', index=0, sep=' ')

# 导入相关库
cat = {
    'model': 'category',
    'brand': 'category',
    'bodyType': 'category',
    'fuelType': 'category',
    'gearbox': 'category',
    'notRepairedDamage': 'category',
    'regionCode': 'category',
}

df = pd.read_csv('./user_data/df.csv', sep=' ', dtype=cat)

for i in range(15):
    for j in range(i+1, 15):
        df['new'+str(i)+'*'+str(j)] = df['v_'+str(i)]*df['v_'+str(j)]


#第二批特征工程
for i in range(15):
    for j in range(i+1, 15):
        df['new'+str(i)+'+'+str(j)] = df['v_'+str(i)]+df['v_'+str(j)]

# 第三批特征工程
for i in range(15):
    df['new' + str(i) + '*power'] = df['v_' + str(i)] * df['power']


#第四批特征工程
for i in range(15):
    for j in range(i+1, 15):
        df['new'+str(i)+'-'+str(j)] = df['v_'+str(i)]-df['v_'+str(j)]


"""
筛选特征
"""
numerical_cols = df.select_dtypes(exclude='object').columns

# initlize a selector
fgs = GBDTSelector()
# fit data
param = {'boosting_type': 'gbdt',
         'num_leaves': 55,  # 过小容易过拟合，过大则计算时间长
         'max_depth': 35,
         "lambda_l2": 2,  # 防止过拟合
         'min_data_in_leaf': 20,  # 防止过拟合，好像都不用怎么调
         'objective': 'regression_l1',
         'learning_rate': 0.02,
         "min_child_samples": 20,
         "feature_fraction": 0.7,
         "bagging_freq": 1,
         "bagging_fraction": 0.9,
         "bagging_seed": 11,
         "metric": 'mae',
         }
train_X = df[df.train == 1].drop(['price', 'SaleID', 'regionCode'], axis=1)
train_y = df[df.train == 1]['price']
train_y_ln = np.log(train_y)
fgs.fit(train_X, train_y_ln, lgb_params=param, eval_ratio=0.2,
        early_stopping_rounds=300, importance_type='gain',
        num_boost_round=15000
        )
gain = pd.DataFrame(fgs.feature_importance)
gain['name'] = train_X.columns
gain = gain.sort_values(0, ascending=False)
df[list(gain.iloc[:60].name)+['price', 'SaleID', 'regionCode', 'train']
   ].to_csv('./user_data/df_s.csv', index=0, sep=' ')


def main(args):
    cat = {
        'model': 'category',
        'brand': 'category',
        'bodyType': 'category',
        'fuelType': 'category',
        'gearbox': 'category',
        'notRepairedDamage': 'category',
    }

    df = pd.read_csv('./user_data/df_s.csv', sep=' ', dtype=cat)
    train_X = df[df.train == 1].drop(['price', 'SaleID', 'regionCode'], axis=1)
    train_y = df[df.train == 1]['price']
    train_y_ln = np.log1p(train_y)

    model = cb.CatBoostRegressor(**args)
    #######用于交叉验证时将以下代码删去#######
    test_X = df[df.train == 0].drop(['price', 'SaleID', 'regionCode'], axis=1)
    model.fit(X=train_X,
              y=train_y_ln,
              cat_features=[i for i in cat.keys() if i in train_X.columns],
              early_stopping_rounds=300,
              verbose=10000)
    p = model.predict(test_X)
    submit = pd.DataFrame()
    submit['SaleID'] = df[df.train == 0].SaleID
    submit['price'] = np.expm1(p)
    submit.to_csv('./prediction_result/submit_cgb.csv', index=0)
    print('end for predict')
    return


params = {
    'n_estimators': 1000000,
    'loss_function': 'MAE',
    'eval_metric': 'MAE',
    'learning_rate': 0.02,  # 设置为0.02会导致训练时间极——————————————长，建议设高一点或者找gpu加速
    'depth': 6,
    # 'use_best_model': True,
    # 'task_type':'GPU'
    'subsample': 0.6,
    'bootstrap_type': 'Bernoulli',
    'reg_lambda': 3,
    'one_hot_max_size': 2,
}
main(params)

def main(args):
    cat = {
        'model': 'category',
        'brand': 'category',
        'bodyType': 'category',
        'fuelType': 'category',
        'gearbox': 'category',
        'notRepairedDamage': 'category',
        'regionCode': 'category',
    }

    df = pd.read_csv('./user_data/df_s.csv', sep=' ', dtype=cat)
    #df['regionCode_count'] = pd.qcut(df.groupby(['regionCode'])['SaleID'].transform('count'), q=10,labels=range(10))
    #df['city'] = pd.Categorical(df['regionCode'].apply(lambda x: str(x)[:2]))

    train_X = df[df.train == 1].drop(['price', 'SaleID', 'regionCode'], axis=1)
    train_y = df[df.train == 1]['price']
    train_y_ln = np.log1p(train_y)

    model = LGBMRegressor(**args)
    '''
    test_X = df[df.train == 0].drop(['price', 'SaleID', 'regionCode'], axis=1)
    model.fit(X=train_X,
              y=train_y_ln)
    p = model.predict(test_X)
    submit = pd.DataFrame()
    submit['SaleID'] = df[df.train == 0].SaleID
    submit['price'] = np.expm1(p)
    submit.to_csv('./prediction_result/submit.csv', index=0)
    print('end for predict')
    return
    '''
    n = 1

    def maee(y_true, y_pred):
        nonlocal n
        loss = mean_absolute_error(np.expm1(y_true), np.expm1(y_pred))
        print(f'mae in fold {n}:{loss}')
        n += 1
        return loss

    mae = cross_val_score(model,
                          X=train_X,
                          y=train_y_ln,
                          verbose=0,
                          cv=5,
                          scoring=make_scorer(maee))
    print(f'{mae}')
    print(f'with mae.mean = {np.mean(mae)}')



param = {'boosting_type': 'gbdt',
             'num_leaves': 55,  # 过小容易过拟合，过大则计算时间长
             'max_depth': 35,
             "lambda_l2": 2,  # 防止过拟合
             'min_data_in_leaf': 20,  # 防止过拟合，好像都不用怎么调
             'objective': 'regression_l1',
             'learning_rate': 0.02,
             "min_child_samples": 20,
             'n_estimators': 15000,
             "feature_fraction": 0.7,
             "bagging_freq": 1,
             "bagging_fraction": 0.9,
             "bagging_seed": 11,
             "metric": 'mae',
             }
main(param)
ls = pd.read_csv('./prediction_result/submit.csv')
cs = pd.read_csv('./prediction_result/submit_cgb.csv')
fs = pd.DataFrame()
fs['SaleID'] = ls.SaleID
fs['price'] = (cs.price+ls.price)/2
fs.to_csv('./prediction_result/submit_fusion.csv', index=0)
